<!DOCTYPE html>
<html  lang="zh-CN" >
    <head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, minimum-scale=1, initial-scale=1, maximum-scale=5, viewport-fit=cover">
    <title>Kudu集群调优建议 | 学为我用</title>
    <meta name="description" content="Kudu Tablet Server Maintenance Threads解释：Kudu后台对数据进行维护操作，如写入数据时的并发线程数，建议的是数据目录的三分之一参数：maintenance_manager_num_threads Kudu Tablet Server Block Cache Capacity Tablet解释：分配给Kudu Tablet Server块缓存的最大内存量，建议">
<meta property="og:type" content="article">
<meta property="og:title" content="Kudu集群调优建议">
<meta property="og:url" content="https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121740-kudutunning/index.html">
<meta property="og:site_name" content="学为我用">
<meta property="og:description" content="Kudu Tablet Server Maintenance Threads解释：Kudu后台对数据进行维护操作，如写入数据时的并发线程数，建议的是数据目录的三分之一参数：maintenance_manager_num_threads Kudu Tablet Server Block Cache Capacity Tablet解释：分配给Kudu Tablet Server块缓存的最大内存量，建议">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2020-10-12T09:45:19.000Z">
<meta property="article:modified_time" content="2020-10-12T10:06:14.109Z">
<meta property="article:author" content="章瑱彬">
<meta property="article:tag" content="大数据">
<meta name="twitter:card" content="summary">

    
    <link rel="icon" href="/zzbomb/images/favicon_128.ico" type="image/x-icon">

    
<link rel="stylesheet" href="/zzbomb/css/common.min.css">



    
    
    
    
        <link href="//cdn.jsdelivr.net/npm/lightgallery.js@1.1.3/dist/css/lightgallery.min.css" rel="stylesheet">
    
    
    
<link rel="stylesheet" href="/zzbomb/css/iconfont.min.css">

    
<meta name="generator" content="Hexo 4.2.1"></head>

    <body>
        <header class="header header-fixture">
    <div class="profile-search-wrap flex sm:block">
        
        
        <div class="profile sm:text-center md:px-1 lg:px-3 sm:pb-4 sm:pt-6">
            <a id="avatar" role="link" href="https://gitee.com/123zhangzhenbin" class="inline-block lg:w-16 lg:h-16 w-8 h-8 m-2" target="_blank" rel="noopener" rel="noreferrer" >
                <img src="/zzbomb/images/g.jpg" class="rounded-full" alt="avatar">
            </a>
            <h2 id="name" class="hidden lg:block">ZZB</h2>
            <h3 id="title" class="hidden xl:block">Java，大数据，人工智能，区块链</h3>
            
            <small id="location" class="hidden lg:block">
                <i class="iconfont icon-map-icon"></i>
                DongGuan, China
            </small>
            
        </div>
        
        
<div class="search flex-1 flex lg:inline-block sm:hidden lg:px-4 lg:mt-2 lg:mb-4 lg:w-full">
    <form id="search-form" class="my-auto flex-1 lg:border lg:border-solid lg:border-gray-200">
        <div class="input-group table bg-gray-100 lg:bg-white w-full">
            <input id="search-input" type="text" placeholder="搜索" class="inline-block w-full bg-gray-100 lg:bg-white">
            <span class="table-cell">
                <button name="search tigger button" disabled>
                    <i class="iconfont icon-search m-2"></i>
                </button>
            </span>
        </div>
    </form>
        
<script id="search-teamplate" type="text/html" data-path="/zzbomb/content.json">
    <div>
        <div class="search-header bg-gray-400">
            <input id="actual-search-input" model="keyword" ref="input" class="inline-block w-full h-10 px-2 py-1" placeholder="搜索" type="text">
        </div>
        <div class="search-result bg-gray-200">
            {{#each searchPosts}}
            <a href="/{{ path }}" class="result-item block px-2 pb-3 mb-1 pt-1 hover:bg-indigo-100">
                <i class="iconfont icon-file"></i>
                <h1 class="result-title inline font-medium text-lg">{{ title }}</h1>
                <p class="result-content text-gray-600 text-sm">{{{ text }}}</p>
            </a>
            {{/each}}
        </div>
    </div>
</script>

</div>


        <button name="menu toogle button" id="menu-toggle-btn" class="block sm:hidden p-3" role="button" aria-expanded="false">
            <i class="iconfont icon-hamburger"></i>
        </button>
    </div>
    <nav id="menu-nav" class="hidden sm:flex flex-col">
        
        
            <div class="menu-item menu-home" role="menuitem">
                <a href="/zzbomb/.">
                    <i class="iconfont icon-home" aria-hidden="true"></i>
                    <span class="menu-title">首页</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-archives" role="menuitem">
                <a href="/zzbomb/archives">
                    <i class="iconfont icon-archive" aria-hidden="true"></i>
                    <span class="menu-title">归档</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-categories" role="menuitem">
                <a href="/zzbomb/categories">
                    <i class="iconfont icon-folder" aria-hidden="true"></i>
                    <span class="menu-title">分类</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-tags" role="menuitem">
                <a href="/zzbomb/tags">
                    <i class="iconfont icon-tag" aria-hidden="true"></i>
                    <span class="menu-title">标签</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-repository" role="menuitem">
                <a href="/zzbomb/repository">
                    <i class="iconfont icon-project" aria-hidden="true"></i>
                    <span class="menu-title">项目</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-links" role="menuitem">
                <a href="/zzbomb/links">
                    <i class="iconfont icon-friend" aria-hidden="true"></i>
                    <span class="menu-title">友链</span>
                </a>
            </div>
        
        
            <div class="menu-item menu-about" role="menuitem">
                <a href="/zzbomb/about">
                    <i class="iconfont icon-cup" aria-hidden="true"></i>
                    <span class="menu-title">关于</span>
                </a>
            </div>
        
        
<div class="social-links flex sm:flex-col lg:hidden mt-5">
    
        <span class="social-item text-center">
            <a href="https://gitee.com/123zhangzhenbin" target="_blank" rel="noopener">
                <i class="iconfont social-icon icon-github"></i>
                <span class="menu-title hidden lg:inline">menu.github</span>
            </a>
        </span>
    
</div>


    </nav>
</header>

        <section class="main-section">
            
    <main class="flex-1 px-4 py-12 md:px-5 lg:px-8 lg:py-4 relative min-h-screen">
    

    <article class="content article article-archives article-type-list" itemscope="">
        <header class="article-header">
            
    
        <h1 class="article-title text-lg" itemprop="name">
            Kudu集群调优建议
        </h1>
    



            <p class="article-meta mb-3 text-xs">
                <span class="article-date">
    <i class="iconfont icon-calendar-check"></i>
	<a href="/zzbomb/2020/10/12/202010121740-kudutunning/" class="article-date">
	  <time datetime="2020-10-12T09:45:19.000Z" itemprop="datePublished">10月 12</time>
	</a>
</span>

                

                
    <span class="article-tags">
    <i class="iconfont icon-tag"></i>
    <a class="article-tag-link" href="/zzbomb/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/" rel="tag">大数据</a>
  </span>


                <span class="_partial/post-comment"><i class="icon icon-comment"></i>
                    <a href="/zzbomb/2020/10/12/202010121740-kudutunning/#comments" class="article-comment-link">
                        评论
                    </a>
                </span>
                

            </p>
        </header>
        <div class="marked-body article-body">
            <h2 id="Kudu-Tablet-Server-Maintenance-Threads"><a href="#Kudu-Tablet-Server-Maintenance-Threads" class="headerlink" title="Kudu Tablet Server Maintenance Threads"></a>Kudu Tablet Server Maintenance Threads</h2><p>解释：Kudu后台对数据进行维护操作，如写入数据时的并发线程数，建议的是数据目录的三分之一<br>参数：maintenance_manager_num_threads</p>
<h2 id="Kudu-Tablet-Server-Block-Cache-Capacity-Tablet"><a href="#Kudu-Tablet-Server-Block-Cache-Capacity-Tablet" class="headerlink" title="Kudu Tablet Server Block Cache Capacity Tablet"></a>Kudu Tablet Server Block Cache Capacity Tablet</h2><p>解释：分配给Kudu Tablet Server块缓存的最大内存量，建议是2-10G<br>参数：block_cache_capacity_mb</p>
<h2 id="Kudu-Tablet-Server-Hard-Memory-Limit-Kudu"><a href="#Kudu-Tablet-Server-Hard-Memory-Limit-Kudu" class="headerlink" title="Kudu Tablet Server Hard Memory Limit Kudu"></a>Kudu Tablet Server Hard Memory Limit Kudu</h2><p>解释：Tablet Server能使用的最大内存量，有多大，设置多大，tabletServer在批量写入数据时并非实时写入磁盘，而是先Cache在内存中，再flush到磁盘。<br>这个值设置过小时，会造成Kudu数据写入性能显著下降。对于写入性能要求比较高的集群，建议设置更大的值（一般是机器内存的百分之80）<br>参数：memory_limit_hard_bytes</p>
<h2 id="其他"><a href="#其他" class="headerlink" title="其他"></a>其他</h2><ul>
<li>建议每个表50columns左右，不能超过300个，公司业务表 大多是20~100个字段之间</li>
<li>设置ntp服务器的时间误差不超过20s（默认是10s）<br>  参数：max_clock_sync_error_usec=20000000</li>
<li>设置rpc的连接时长（默认是3s，建议不要设置）<br>  参数：–rpc_negotiation_timeout_ms=300000</li>
<li>设置rpc一致性选择的连接时长（默认为1s，建议不要设置）<br>  参数：–consensus_rpc_timeout_ms=1000</li>
<li>记录kudu的crash的信息<br>  Kudu在Kudu遇到崩溃时，使用GoogleBreakpad库来生成minidump。这些minidumps的大小通常只有几MB，即使禁用了核心转储生成，也会生成，生成minidumps只能在Linux上建立。minidump文件包含有关崩溃的进程的重要调试信息，包括加载的共享库及其版本，崩溃时运行的线程列表，处理器寄存器的状态和每个线程的堆栈内存副本，以及CPU和操作系统版本信息。<br>  Minitump可以通过电子邮件发送给Kudu开发人员或附加到JIRA，以帮助Kudu开发人员调试崩溃。为了使其有用，开发人员将需要知道Kudu的确切版本和发生崩溃的操作系统。<br>  请注意，虽然切版本和发生崩溃的操作系统。请注意，虽然minidump不包含堆内存转储，但它确实包含堆栈内存，因此可以将应用程序数据显示在minidump中。如果机密或个人信息存储在群集上，请不要共享minidump文件。<br>  参数：–minidump_path=minidumps –max_minidumps=9</li>
<li>在开发过程中，难免会手误，由于在通过impala-shell建kudu表时候把bigint类型的字段写成了string，以致后面在计算时候报错。但由于该表有2亿的数据（交易明细表），不可能重新抽取，于是按照关系型数据库的思维来操作。<ol>
<li>暂以A表示原表，新建一个正确的表B，</li>
<li>insert into B select * from A;(此步耗时2-3分钟,与机器性能有关)</li>
<li>然后drop A或者rename A to C.d alter table B rename to A以上步骤完成后，在关系型数据（MySQL,Oracle等）是没问题的。<br>通过impala-shell也是可以查询到数据，但是后面程序在通过Kuducontext读取kudu表数据的时候，一直读取不到数据,也不会报错。这个问题纠结了很久，于是去查阅了官方文档。<br>有如下说明：Altering table properties only changes Impala’s metadata about the table, not the underlying table itself. These statements do notmodify any Kudu data.就是说,在impala-shell里面的alter table操作，只会更改impala的元数据，而不会更改任何kudu的数据。<br>可以通过ALTER TABLE B SET TBLPROPERTIES(‘kudu.table_name’ = ‘A’);<br>但是在impala-shell里面执行droptable操作，会删除kudu的表和kudu表里面的数据，同时也会删除kudu和impala的映射关系及impala元数据里面的表信息。所以，如果遇到类似错误信息，可以drop table，再新建表，或者使用上述语法改名。<br>最好是先修改impala元数据(alter table B rename to A)，再执行ALTER TABLE B SET TBLPROPERTIES(‘kudu.table_name’ = ‘A’);</li>
</ol>
</li>
<li>如果tabletServer节点数小于存储副本数， 建表会有问题<br>  CAUSED BY: NonRecoverableException: Not enough live tablet servers to create a table with the requested replication factor 3. 1 tabletservers are alive.</li>
<li>kudu表最好不要做任何压缩，保证原始扫描性能发挥最好；假如对查询性能要求比存储要求高的话；大部分企业对实时查询效率要求高，而且存储成本毕竟低。</li>
<li>查询慢的sql，一般要拿出来；方便的话做下explain，看下kudu有没有过滤部分数据关键字kudupredicates；假如sql没问题，那在impala-shell执行这个sql，最后执行summray命令，重点查看单点峰值内存和时间比较大的点，对相关的表做优化，解决数据倾斜问题。</li>
<li>大表最好不要delete，直接drop，然后create；磁盘空间会释放的。</li>
</ul>

        </div>
        
<blockquote class="copyright">
    <p><strong>本文链接 : </strong><a class="permalink" href="https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121740-kudutunning/">https://123zhangzhenbin.gitee.io/zzbomb/2020/10/12/202010121740-kudutunning/</a></p>
    <p><strong>This article is available under <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank" rel="noopener noreferrer">Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)</a> License</strong></p>
</blockquote>


    </article>
    
    <section id="comments">
        
    </section>


    

</main>


<aside style="" id="sidebar" class="aside aside-fixture">
    <div class="toc-sidebar">
        <nav id="toc" class="article-toc">
            <h3 class="toc-title">文章目录</h3>
            <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#Kudu-Tablet-Server-Maintenance-Threads"><span class="toc-number">1.</span> <span class="toc-text">Kudu Tablet Server Maintenance Threads</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Kudu-Tablet-Server-Block-Cache-Capacity-Tablet"><span class="toc-number">2.</span> <span class="toc-text">Kudu Tablet Server Block Cache Capacity Tablet</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Kudu-Tablet-Server-Hard-Memory-Limit-Kudu"><span class="toc-number">3.</span> <span class="toc-text">Kudu Tablet Server Hard Memory Limit Kudu</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#其他"><span class="toc-number">4.</span> <span class="toc-text">其他</span></a></li></ol>
        </nav>
    </div>
</aside>





        </section>
        <footer class="hidden lg:block fixed bottom-0 left-0 sm:w-1/12 lg:w-1/6 bg-gray-100 z-40">
    
    <div class="footer-social-links">
        
            <a href="https://gitee.com/123zhangzhenbin" target="_blank" rel="noopener">
                <i class="iconfont icon-github"></i>
            </a>
        
    </div>
    
    <p style="display: block;text-align: center;font-size: 13px;"><a href="https://beian.miit.gov.cn/" target="_blank">粤ICP备16013026号</a></p>
    
        <p class="footer-custom"><a href="https://beian.miit.gov.cn/" target="_blank">粤ICP备16013026号</a></p>
        <p class="theme-brand">Theme by <a href="https://github.com/fengkx/hexo-theme-purer" target="_blank" rel="nofollow noopener noreferrer noopener">hexo-theme-purer</a></p>

    
</footer>

        <div id="mask" class="hidden mask fixed inset-0 bg-gray-900 opacity-75 z-40"></div>
        <div id="search-view-container" class="hidden shadow-xl"></div>
        
<script src="/zzbomb/js/dom-event.min.js"></script>

<script src="//cdn.jsdelivr.net/npm/yox@1.0.0-alpha.121/dist/standard/prod/yox.min.js"></script>


<script src="/zzbomb/js/search.min.js"></script>



    <script src="//cdn.jsdelivr.net/npm/lightgallery.js@1.1.3/dist/js/lightgallery.min.js"></script>
    
<script src="/zzbomb/js/light-gallery.min.js"></script>





    </body>
</html>
